
import java.util.Random;

@SuppressWarnings("rawtypes")
public class AdvicedQLearningDriver extends Driver {


    //previous state
    private Vertex prev_intersection;

    private QTable qtable;

    public AdvicedQLearningDriver(long id, Vertex origin, Vertex destination, Graph G) {
        super(id, origin, destination, G);

        resetAll();

    }

    public void reset() {
        super.reset();
        prev_intersection = null;
    }

    public void resetAll() {
        super.resetAll();
        qtable = new QTable(G.getV());
    }

    @SuppressWarnings("unchecked")
    @Override
    public void step_A() {
        curr_road = epsilonGreedy();
        takeAction(curr_road);
        //System.out.println(this + ": I'm at " + curr_intersection);
    }

    @SuppressWarnings("unchecked")
    public void step_B() {
        route.add(new RouteEntry(curr_road, curr_road.getTravelTime()));
        updateQTable();
    }

    private Edge chooseAction() {
		//TODO advice
        //TODO state object
        return chooseActionRandomly();
    }

    private Edge epsilonGreedy() {

        if (random.nextDouble() < Params.EG_EPSILON) {
            return chooseActionRandomly();
        } else {
            return chooseActionGreedily();
        }

    }

    private Edge chooseActionRandomly() {
        return curr_intersection.getOutEdges().get(random.nextInt(curr_intersection.getOutEdges().size()));
    }

    private Edge chooseActionGreedily() {
        Edge choosen_a = curr_intersection.getOutEdges().get(0);
        for (Edge a : curr_intersection.getOutEdges()) {
            if (qtable.getQValue(curr_intersection, a) > qtable.getQValue(curr_intersection, choosen_a)) {
                choosen_a = a;
            }
        }
        return choosen_a;
    }

    @SuppressWarnings("unchecked")
    private void takeAction(Edge action) {

        prev_intersection = curr_intersection;
        curr_intersection = action.getTo();

    }

    /**
     * Update Q-Table according to the equation Q(s,a) = Q(s,a) + alpha * (r +
     * gamma * max_a'(Q(s',a')) - Q(s,a).
     */
    private void updateQTable() {

        float qsa = qtable.getQValue(prev_intersection, curr_road);
        double r = curr_road.getTravelTime();
        double max_qsa_new = qtable.getQValue(curr_intersection, curr_intersection.getOutEdges().get(0));

        for (Edge e : curr_intersection.getOutEdges()) {
            if (qtable.getQValue(curr_intersection, e) > max_qsa_new) {
                max_qsa_new = qtable.getQValue(curr_intersection, e);
            }
        }

		//qsa = qsa + Params.QL_ALPHA * (r + Params.QL_GAMMA * max_qsa_new - qsa);
        qtable.setQValue(prev_intersection, curr_road, qsa);

    }

    public String toString() {
        return "QL" + super.toString();
    }

    public void printQTable() {
        qtable.print();
    }

}
